# pip3 install transformers
# python3 deepseek_tokenizer.py
import transformers

chat_tokenizer_dir = "./"

tokenizer = transformers.AutoTokenizer.from_pretrained(chat_tokenizer_dir, trust_remote_code=True)

def get_token_length(file):
    file_content = file.read().decode("utf-8")
    result = tokenizer.decode(file_content)
    return len(result)
